agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v14 5/8] Row pattern recognition patch (executor). 330+ messages / 2 participants [nested] [flat]
* [PATCH v14 5/8] Row pattern recognition patch (executor). @ 2024-02-28 13:59 Tatsuo Ishii <ishii@postgresql.org> 0 siblings, 0 replies; 330+ messages in thread From: Tatsuo Ishii @ 2024-02-28 13:59 UTC (permalink / raw) --- src/backend/executor/nodeWindowAgg.c | 1617 +++++++++++++++++++++++++- src/backend/utils/adt/windowfuncs.c | 37 +- src/include/catalog/pg_proc.dat | 6 + src/include/nodes/execnodes.h | 30 + 4 files changed, 1679 insertions(+), 11 deletions(-) diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index 62d028379b..c473cbd218 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -36,6 +36,7 @@ #include "access/htup_details.h" #include "catalog/objectaccess.h" #include "catalog/pg_aggregate.h" +#include "catalog/pg_collation_d.h" #include "catalog/pg_proc.h" #include "executor/executor.h" #include "executor/nodeWindowAgg.h" @@ -48,6 +49,7 @@ #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" +#include "utils/fmgroids.h" #include "utils/expandeddatum.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -159,6 +161,43 @@ typedef struct WindowStatePerAggData bool restart; /* need to restart this agg in this cycle? */ } WindowStatePerAggData; +/* + * Set of StringInfo. Used in RPR. + */ +typedef struct StringSet +{ + StringInfo *str_set; + Size set_size; /* current array allocation size in number of + * items */ + int set_index; /* current used size */ +} StringSet; + +/* + * Allowed subsequent PATTERN variables positions. + * Used in RPR. + * + * pos represents the pattern variable defined order in DEFINE caluase. For + * example. "DEFINE START..., UP..., DOWN ..." and "PATTERN START UP DOWN UP" + * will create: + * VariablePos[0].pos[0] = 0; START + * VariablePos[1].pos[0] = 1; UP + * VariablePos[1].pos[1] = 3; UP + * VariablePos[2].pos[0] = 2; DOWN + * + * Note that UP has two pos because UP appears in PATTERN twice. + * + * By using this strucrture, we can know which pattern variable can be followed + * by which pattern variable(s). For example, START can be followed by UP and + * DOWN since START's pos is 0, and UP's pos is 1 or 3, DOWN's pos is 2. + * DOWN can be followed by UP since UP's pos is either 1 or 3. + * + */ +#define NUM_ALPHABETS 26 /* we allow [a-z] variable initials */ +typedef struct VariablePos +{ + int pos[NUM_ALPHABETS]; /* postion(s) in PATTERN */ +} VariablePos; + static void initialize_windowaggregate(WindowAggState *winstate, WindowStatePerFunc perfuncstate, WindowStatePerAgg peraggstate); @@ -184,6 +223,7 @@ static void release_partition(WindowAggState *winstate); static int row_is_in_frame(WindowAggState *winstate, int64 pos, TupleTableSlot *slot); + static void update_frameheadpos(WindowAggState *winstate); static void update_frametailpos(WindowAggState *winstate); static void update_grouptailpos(WindowAggState *winstate); @@ -195,9 +235,48 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype); static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1, TupleTableSlot *slot2); + +static int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot, + int relpos, int seektype, bool set_mark, + bool *isnull, bool *isout); static bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot); +static void attno_map(Node *node); +static bool attno_map_walker(Node *node, void *context); +static int row_is_in_reduced_frame(WindowObject winobj, int64 pos); +static bool rpr_is_defined(WindowAggState *winstate); + +static void create_reduced_frame_map(WindowAggState *winstate); +static int get_reduced_frame_map(WindowAggState *winstate, int64 pos); +static void register_reduced_frame_map(WindowAggState *winstate, int64 pos, + int val); +static void clear_reduced_frame_map(WindowAggState *winstate); +static void update_reduced_frame(WindowObject winobj, int64 pos); + +static int64 evaluate_pattern(WindowObject winobj, int64 current_pos, + char *vname, StringInfo encoded_str, bool *result); + +static bool get_slots(WindowObject winobj, int64 current_pos); + +static int search_str_set(char *pattern, StringSet * str_set, + VariablePos * variable_pos); +static char pattern_initial(WindowAggState *winstate, char *vname); +static int do_pattern_match(char *pattern, char *encoded_str); + +static StringSet * string_set_init(void); +static void string_set_add(StringSet * string_set, StringInfo str); +static StringInfo string_set_get(StringSet * string_set, int index); +static int string_set_get_size(StringSet * string_set); +static void string_set_discard(StringSet * string_set); +static VariablePos * variable_pos_init(void); +static void variable_pos_register(VariablePos * variable_pos, char initial, + int pos); +static bool variable_pos_compare(VariablePos * variable_pos, + char initial1, char initial2); +static int variable_pos_fetch(VariablePos * variable_pos, char initial, + int index); +static void variable_pos_discard(VariablePos * variable_pos); /* * initialize_windowaggregate @@ -673,6 +752,7 @@ eval_windowaggregates(WindowAggState *winstate) WindowObject agg_winobj; TupleTableSlot *agg_row_slot; TupleTableSlot *temp_slot; + bool agg_result_isnull; numaggs = winstate->numaggs; if (numaggs == 0) @@ -778,6 +858,9 @@ eval_windowaggregates(WindowAggState *winstate) * Note that we don't strictly need to restart in the last case, but if * we're going to remove all rows from the aggregation anyway, a restart * surely is faster. + * + * - if RPR is enabled and skip mode is SKIP TO NEXT ROW, + * we restart aggregation too. *---------- */ numaggs_restart = 0; @@ -788,8 +871,11 @@ eval_windowaggregates(WindowAggState *winstate) (winstate->aggregatedbase != winstate->frameheadpos && !OidIsValid(peraggstate->invtransfn_oid)) || (winstate->frameOptions & FRAMEOPTION_EXCLUSION) || - winstate->aggregatedupto <= winstate->frameheadpos) + winstate->aggregatedupto <= winstate->frameheadpos || + (rpr_is_defined(winstate) && + winstate->rpSkipTo == ST_NEXT_ROW)) { + elog(DEBUG1, "peraggstate->restart is set"); peraggstate->restart = true; numaggs_restart++; } @@ -862,7 +948,22 @@ eval_windowaggregates(WindowAggState *winstate) * head, so that tuplestore can discard unnecessary rows. */ if (agg_winobj->markptr >= 0) - WinSetMarkPosition(agg_winobj, winstate->frameheadpos); + { + int64 markpos = winstate->frameheadpos; + + if (rpr_is_defined(winstate)) + { + /* + * If RPR is used, it is possible PREV wants to look at the + * previous row. So the mark pos should be frameheadpos - 1 + * unless it is below 0. + */ + markpos -= 1; + if (markpos < 0) + markpos = 0; + } + WinSetMarkPosition(agg_winobj, markpos); + } /* * Now restart the aggregates that require it. @@ -917,6 +1018,30 @@ eval_windowaggregates(WindowAggState *winstate) { winstate->aggregatedupto = winstate->frameheadpos; ExecClearTuple(agg_row_slot); + + /* + * If RPR is defined, we do not use aggregatedupto_nonrestarted. To + * avoid assertion failure below, we reset aggregatedupto_nonrestarted + * to frameheadpos. + */ + if (rpr_is_defined(winstate)) + aggregatedupto_nonrestarted = winstate->frameheadpos; + } + + agg_result_isnull = false; + /* RPR is defined? */ + if (rpr_is_defined(winstate)) + { + /* + * If the skip mode is SKIP TO PAST LAST ROW and we already know that + * current row is a skipped row, we don't need to accumulate rows, + * just return NULL. Note that for unamtched row, we need to do + * aggregation so that count(*) shows 0, rather than NULL. + */ + if (winstate->rpSkipTo == ST_PAST_LAST_ROW && + get_reduced_frame_map(winstate, + winstate->currentpos) == RF_SKIPPED) + agg_result_isnull = true; } /* @@ -930,6 +1055,12 @@ eval_windowaggregates(WindowAggState *winstate) { int ret; + elog(DEBUG1, "===== loop in frame starts: " INT64_FORMAT, + winstate->aggregatedupto); + + if (agg_result_isnull) + break; + /* Fetch next row if we didn't already */ if (TupIsNull(agg_row_slot)) { @@ -945,9 +1076,32 @@ eval_windowaggregates(WindowAggState *winstate) ret = row_is_in_frame(winstate, winstate->aggregatedupto, agg_row_slot); if (ret < 0) break; + if (ret == 0) goto next_tuple; + if (rpr_is_defined(winstate)) + { + /* + * If the row status at currentpos is already decided and current + * row status is not decided yet, it means we passed the last + * reduced frame. Time to break the loop. + */ + if (get_reduced_frame_map(winstate, + winstate->currentpos) != RF_NOT_DETERMINED && + get_reduced_frame_map(winstate, + winstate->aggregatedupto) == RF_NOT_DETERMINED) + break; + + /* + * Otherwise we need to calculate the reduced frame. + */ + ret = row_is_in_reduced_frame(winstate->agg_winobj, + winstate->aggregatedupto); + if (ret == -1) /* unmatched row */ + break; + } + /* Set tuple context for evaluation of aggregate arguments */ winstate->tmpcontext->ecxt_outertuple = agg_row_slot; @@ -976,6 +1130,7 @@ next_tuple: ExecClearTuple(agg_row_slot); } + /* The frame's end is not supposed to move backwards, ever */ Assert(aggregatedupto_nonrestarted <= winstate->aggregatedupto); @@ -996,6 +1151,16 @@ next_tuple: peraggstate, result, isnull); + /* + * RPR is defined and we just return NULL because skip mode is SKIP TO + * PAST LAST ROW and current row is skipped row. + */ + if (agg_result_isnull) + { + *isnull = true; + *result = (Datum) 0; + } + /* * save the result in case next row shares the same frame. * @@ -1090,6 +1255,7 @@ begin_partition(WindowAggState *winstate) winstate->framehead_valid = false; winstate->frametail_valid = false; winstate->grouptail_valid = false; + create_reduced_frame_map(winstate); winstate->spooled_rows = 0; winstate->currentpos = 0; winstate->frameheadpos = 0; @@ -2053,6 +2219,11 @@ ExecWindowAgg(PlanState *pstate) CHECK_FOR_INTERRUPTS(); +#ifdef RPR_DEBUG + elog(DEBUG1, "ExecWindowAgg called. pos: " INT64_FORMAT, + winstate->currentpos); +#endif + if (winstate->status == WINDOWAGG_DONE) return NULL; @@ -2221,6 +2392,17 @@ ExecWindowAgg(PlanState *pstate) /* don't evaluate the window functions when we're in pass-through mode */ if (winstate->status == WINDOWAGG_RUN) { + /* + * If RPR is defined and skip mode is next row, we need to clear + * existing reduced frame info so that we newly calculate the info + * starting from current row. + */ + if (rpr_is_defined(winstate)) + { + if (winstate->rpSkipTo == ST_NEXT_ROW) + clear_reduced_frame_map(winstate); + } + /* * Evaluate true window functions */ @@ -2388,6 +2570,9 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) TupleDesc scanDesc; ListCell *l; + TargetEntry *te; + Expr *expr; + /* check for unsupported flags */ Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK))); @@ -2483,6 +2668,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc, &TTSOpsMinimalTuple); + winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc, + &TTSOpsMinimalTuple); + + winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc, + &TTSOpsMinimalTuple); + + winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc, + &TTSOpsMinimalTuple); + winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot); + /* * create frame head and tail slots only if needed (must create slots in * exactly the same cases that update_frameheadpos and update_frametailpos @@ -2667,6 +2862,43 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) winstate->inRangeAsc = node->inRangeAsc; winstate->inRangeNullsFirst = node->inRangeNullsFirst; + /* Set up SKIP TO type */ + winstate->rpSkipTo = node->rpSkipTo; + /* Set up row pattern recognition PATTERN clause */ + winstate->patternVariableList = node->patternVariable; + winstate->patternRegexpList = node->patternRegexp; + + /* Set up row pattern recognition DEFINE clause */ + winstate->defineInitial = node->defineInitial; + winstate->defineVariableList = NIL; + winstate->defineClauseList = NIL; + if (node->defineClause != NIL) + { + /* + * Tweak arg var of PREV/NEXT so that it refers to scan/inner slot. + */ + foreach(l, node->defineClause) + { + char *name; + ExprState *exps; + + te = lfirst(l); + name = te->resname; + expr = te->expr; + +#ifdef RPR_DEBUG + elog(DEBUG1, "defineVariable name: %s", name); +#endif + winstate->defineVariableList = + lappend(winstate->defineVariableList, + makeString(pstrdup(name))); + attno_map((Node *) expr); + exps = ExecInitExpr(expr, (PlanState *) winstate); + winstate->defineClauseList = + lappend(winstate->defineClauseList, exps); + } + } + winstate->all_first = true; winstate->partition_spooled = false; winstate->more_partitions = false; @@ -2674,6 +2906,64 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) return winstate; } +/* + * Rewrite varno of Var node that is the argument of PREV/NET so that it sees + * scan tuple (PREV) or inner tuple (NEXT). + */ +static void +attno_map(Node *node) +{ + (void) expression_tree_walker(node, attno_map_walker, NULL); +} + +static bool +attno_map_walker(Node *node, void *context) +{ + FuncExpr *func; + int nargs; + Expr *expr; + Var *var; + + if (node == NULL) + return false; + + if (IsA(node, FuncExpr)) + { + func = (FuncExpr *) node; + + if (func->funcid == F_PREV || func->funcid == F_NEXT) + { + /* sanity check */ + nargs = list_length(func->args); + if (list_length(func->args) != 1) + elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", + func->funcid, nargs); + + expr = (Expr *) lfirst(list_head(func->args)); + if (!IsA(expr, Var)) + elog(ERROR, "PREV/NEXT's arg is not Var"); /* XXX: is it possible + * that arg type is + * Const? */ + var = (Var *) expr; + + if (func->funcid == F_PREV) + + /* + * Rewrite varno from OUTER_VAR to regular var no so that the + * var references scan tuple. + */ + var->varno = var->varnosyn; + else + var->varno = INNER_VAR; + +#ifdef RPR_DEBUG + elog(DEBUG1, "PREV/NEXT's varno is rewritten to: %d", var->varno); +#endif + } + } + return expression_tree_walker(node, attno_map_walker, NULL); +} + /* ----------------- * ExecEndWindowAgg * ----------------- @@ -2723,6 +3013,8 @@ ExecReScanWindowAgg(WindowAggState *node) ExecClearTuple(node->agg_row_slot); ExecClearTuple(node->temp_slot_1); ExecClearTuple(node->temp_slot_2); + ExecClearTuple(node->prev_slot); + ExecClearTuple(node->next_slot); if (node->framehead_slot) ExecClearTuple(node->framehead_slot); if (node->frametail_slot) @@ -3083,7 +3375,8 @@ window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot) return false; if (pos < winobj->markpos) - elog(ERROR, "cannot fetch row before WindowObject's mark position"); + elog(ERROR, "cannot fetch row: " INT64_FORMAT " before WindowObject's mark position: " INT64_FORMAT, + pos, winobj->markpos); oldcontext = MemoryContextSwitchTo(winstate->ss.ps.ps_ExprContext->ecxt_per_query_memory); @@ -3403,14 +3696,54 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, WindowAggState *winstate; ExprContext *econtext; TupleTableSlot *slot; - int64 abs_pos; - int64 mark_pos; Assert(WindowObjectIsValid(winobj)); winstate = winobj->winstate; econtext = winstate->ss.ps.ps_ExprContext; slot = winstate->temp_slot_1; + if (WinGetSlotInFrame(winobj, slot, + relpos, seektype, set_mark, + isnull, isout) == 0) + { + econtext->ecxt_outertuple = slot; + return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno), + econtext, isnull); + } + + if (isout) + *isout = true; + *isnull = true; + return (Datum) 0; +} + +/* + * WinGetSlotInFrame + * slot: TupleTableSlot to store the result + * relpos: signed rowcount offset from the seek position + * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL + * set_mark: If the row is found/in frame and set_mark is true, the mark is + * moved to the row as a side-effect. + * isnull: output argument, receives isnull status of result + * isout: output argument, set to indicate whether target row position + * is out of frame (can pass NULL if caller doesn't care about this) + * + * Returns 0 if we successfullt got the slot. false if out of frame. + * (also isout is set) + */ +static int +WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot, + int relpos, int seektype, bool set_mark, + bool *isnull, bool *isout) +{ + WindowAggState *winstate; + int64 abs_pos; + int64 mark_pos; + int num_reduced_frame; + + Assert(WindowObjectIsValid(winobj)); + winstate = winobj->winstate; + switch (seektype) { case WINDOW_SEEK_CURRENT: @@ -3477,11 +3810,25 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, winstate->frameOptions); break; } + num_reduced_frame = row_is_in_reduced_frame(winobj, + winstate->frameheadpos); + if (num_reduced_frame < 0) + goto out_of_frame; + else if (num_reduced_frame > 0) + if (relpos >= num_reduced_frame) + goto out_of_frame; break; case WINDOW_SEEK_TAIL: /* rejecting relpos > 0 is easy and simplifies code below */ if (relpos > 0) goto out_of_frame; + + /* + * RPR cares about frame head pos. Need to call + * update_frameheadpos + */ + update_frameheadpos(winstate); + update_frametailpos(winstate); abs_pos = winstate->frametailpos - 1 + relpos; @@ -3548,6 +3895,14 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, mark_pos = 0; /* keep compiler quiet */ break; } + + num_reduced_frame = row_is_in_reduced_frame(winobj, + winstate->frameheadpos + relpos); + if (num_reduced_frame < 0) + goto out_of_frame; + else if (num_reduced_frame > 0) + abs_pos = winstate->frameheadpos + relpos + + num_reduced_frame - 1; break; default: elog(ERROR, "unrecognized window seek type: %d", seektype); @@ -3566,15 +3921,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, *isout = false; if (set_mark) WinSetMarkPosition(winobj, mark_pos); - econtext->ecxt_outertuple = slot; - return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno), - econtext, isnull); + return 0; out_of_frame: if (isout) *isout = true; *isnull = true; - return (Datum) 0; + return -1; } /* @@ -3605,3 +3958,1249 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull) return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno), econtext, isnull); } + +/* + * rpr_is_defined + * return true if Row pattern recognition is defined. + */ +static +bool +rpr_is_defined(WindowAggState *winstate) +{ + return winstate->patternVariableList != NIL; +} + +/* + * row_is_in_reduced_frame + * Determine whether a row is in the current row's reduced window frame + * according to row pattern matching + * + * The row must has been already determined that it is in a full window frame + * and fetched it into slot. + * + * Returns: + * = 0, RPR is not defined. + * >0, if the row is the first in the reduced frame. Return the number of rows + * in the reduced frame. + * -1, if the row is unmatched row + * -2, if the row is in the reduced frame but needed to be skipped because of + * AFTER MATCH SKIP PAST LAST ROW + */ +static +int +row_is_in_reduced_frame(WindowObject winobj, int64 pos) +{ + WindowAggState *winstate = winobj->winstate; + int state; + int rtn; + + if (!rpr_is_defined(winstate)) + { + /* + * RPR is not defined. Assume that we are always in the the reduced + * window frame. + */ + rtn = 0; +#ifdef RPR_DEBUG + elog(DEBUG1, "row_is_in_reduced_frame returns %d: pos: " INT64_FORMAT, + rtn, pos); +#endif + return rtn; + } + + state = get_reduced_frame_map(winstate, pos); + + if (state == RF_NOT_DETERMINED) + { + update_frameheadpos(winstate); + update_reduced_frame(winobj, pos); + } + + state = get_reduced_frame_map(winstate, pos); + + switch (state) + { + int64 i; + int num_reduced_rows; + + case RF_FRAME_HEAD: + num_reduced_rows = 1; + for (i = pos + 1; + get_reduced_frame_map(winstate, i) == RF_SKIPPED; i++) + num_reduced_rows++; + rtn = num_reduced_rows; + break; + + case RF_SKIPPED: + rtn = -2; + break; + + case RF_UNMATCHED: + rtn = -1; + break; + + default: + elog(ERROR, "Unrecognized state: %d at: " INT64_FORMAT, + state, pos); + break; + } + +#ifdef RPR_DEBUG + elog(DEBUG1, "row_is_in_reduced_frame returns %d: pos: " INT64_FORMAT, + rtn, pos); +#endif + return rtn; +} + +#define REDUCED_FRAME_MAP_INIT_SIZE 1024L + +/* + * create_reduced_frame_map + * Create reduced frame map + */ +static +void +create_reduced_frame_map(WindowAggState *winstate) +{ + winstate->reduced_frame_map = + MemoryContextAlloc(winstate->partcontext, + REDUCED_FRAME_MAP_INIT_SIZE); + winstate->alloc_sz = REDUCED_FRAME_MAP_INIT_SIZE; + clear_reduced_frame_map(winstate); +} + +/* + * clear_reduced_frame_map + * Clear reduced frame map + */ +static +void +clear_reduced_frame_map(WindowAggState *winstate) +{ + Assert(winstate->reduced_frame_map != NULL); + MemSet(winstate->reduced_frame_map, RF_NOT_DETERMINED, + winstate->alloc_sz); +} + +/* + * get_reduced_frame_map + * Get reduced frame map specified by pos + */ +static +int +get_reduced_frame_map(WindowAggState *winstate, int64 pos) +{ + Assert(winstate->reduced_frame_map != NULL); + + if (pos < 0 || pos >= winstate->alloc_sz) + elog(ERROR, "wrong pos: " INT64_FORMAT, pos); + + return winstate->reduced_frame_map[pos]; +} + +/* + * register_reduced_frame_map + * Add/replace reduced frame map member at pos. + * If there's no enough space, expand the map. + */ +static +void +register_reduced_frame_map(WindowAggState *winstate, int64 pos, int val) +{ + int64 realloc_sz; + + Assert(winstate->reduced_frame_map != NULL); + + if (pos < 0) + elog(ERROR, "wrong pos: " INT64_FORMAT, pos); + + if (pos > winstate->alloc_sz - 1) + { + realloc_sz = winstate->alloc_sz * 2; + + winstate->reduced_frame_map = + repalloc(winstate->reduced_frame_map, realloc_sz); + + MemSet(winstate->reduced_frame_map + winstate->alloc_sz, + RF_NOT_DETERMINED, realloc_sz - winstate->alloc_sz); + + winstate->alloc_sz = realloc_sz; + } + + winstate->reduced_frame_map[pos] = val; +} + +/* + * update_reduced_frame + * Update reduced frame info. + */ +static +void +update_reduced_frame(WindowObject winobj, int64 pos) +{ + WindowAggState *winstate = winobj->winstate; + ListCell *lc1, + *lc2; + bool expression_result; + int num_matched_rows; + int64 original_pos; + bool anymatch; + StringInfo encoded_str; + StringInfo pattern_str = makeStringInfo(); + StringSet *str_set; + int initial_index; + VariablePos *variable_pos; + bool greedy = false; + int64 result_pos, + i; + + /* + * Set of pattern variables evaluated to true. Each character corresponds + * to pattern variable. Example: str_set[0] = "AB"; str_set[1] = "AC"; In + * this case at row 0 A and B are true, and A and C are true in row 1. + */ + + /* initialize pattern variables set */ + str_set = string_set_init(); + + /* save original pos */ + original_pos = pos; + + /* + * Check if the pattern does not include any greedy quantifier. If it does + * not, we can just apply the pattern to each row. If it succeeds, we are + * done. + */ + foreach(lc1, winstate->patternRegexpList) + { + char *quantifier = strVal(lfirst(lc1)); + + if (*quantifier == '+' || *quantifier == '*') + { + greedy = true; + break; + } + } + + /* + * Non greedy case + */ + if (!greedy) + { + num_matched_rows = 0; + + foreach(lc1, winstate->patternVariableList) + { + char *vname = strVal(lfirst(lc1)); + + encoded_str = makeStringInfo(); + +#ifdef RPR_DEBUG + elog(DEBUG1, "pos: " INT64_FORMAT " pattern vname: %s", + pos, vname); +#endif + expression_result = false; + + /* evaluate row pattern against current row */ + result_pos = evaluate_pattern(winobj, pos, vname, + encoded_str, &expression_result); + if (!expression_result || result_pos < 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "expression result is false or out of frame"); +#endif + register_reduced_frame_map(winstate, original_pos, + RF_UNMATCHED); + return; + } + /* move to next row */ + pos++; + num_matched_rows++; + } +#ifdef RPR_DEBUG + elog(DEBUG1, "pattern matched"); +#endif + register_reduced_frame_map(winstate, original_pos, RF_FRAME_HEAD); + + for (i = original_pos + 1; i < original_pos + num_matched_rows; i++) + { + register_reduced_frame_map(winstate, i, RF_SKIPPED); + } + return; + } + + /* + * Greedy quantifiers included. Loop over until none of pattern matches or + * encounters end of frame. + */ + for (;;) + { + result_pos = -1; + + /* + * Loop over each PATTERN variable. + */ + anymatch = false; + encoded_str = makeStringInfo(); + + forboth(lc1, winstate->patternVariableList, lc2, + winstate->patternRegexpList) + { + char *vname = strVal(lfirst(lc1)); +#ifdef RPR_DEBUG + char *quantifier = strVal(lfirst(lc2)); + + elog(DEBUG1, "pos: " INT64_FORMAT " pattern vname: %s quantifier: %s", + pos, vname, quantifier); +#endif + expression_result = false; + + /* evaluate row pattern against current row */ + result_pos = evaluate_pattern(winobj, pos, vname, + encoded_str, &expression_result); + if (expression_result) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "expression result is true"); +#endif + anymatch = true; + } + + /* + * If out of frame, we are done. + */ + if (result_pos < 0) + break; + } + + if (!anymatch) + { + /* none of patterns matched. */ + break; + } + + string_set_add(str_set, encoded_str); + +#ifdef RPR_DEBUG + elog(DEBUG1, "pos: " INT64_FORMAT " encoded_str: %s", + encoded_str->data); +#endif + + /* move to next row */ + pos++; + + if (result_pos < 0) + { + /* out of frame */ + break; + } + } + + if (string_set_get_size(str_set) == 0) + { + /* no match found in the first row */ + register_reduced_frame_map(winstate, original_pos, RF_UNMATCHED); + return; + } + +#ifdef RPR_DEBUG + elog(DEBUG2, "pos: " INT64_FORMAT " encoded_str: %s", + pos, encoded_str->data); +#endif + + /* build regular expression */ + pattern_str = makeStringInfo(); + appendStringInfoChar(pattern_str, '^'); + initial_index = 0; + + variable_pos = variable_pos_init(); + + forboth(lc1, winstate->patternVariableList, + lc2, winstate->patternRegexpList) + { + char *vname = strVal(lfirst(lc1)); + char *quantifier = strVal(lfirst(lc2)); + char initial; + + initial = pattern_initial(winstate, vname); + Assert(initial != 0); + appendStringInfoChar(pattern_str, initial); + if (quantifier[0]) + appendStringInfoChar(pattern_str, quantifier[0]); + + /* + * Register the initial at initial_index. If the initial appears more + * than once, all of it's initial_index will be recorded. This could + * happen if a pattern variable appears in the PATTERN clause more + * than once like "UP DOWN UP" "UP UP UP". + */ + variable_pos_register(variable_pos, initial, initial_index); + + initial_index++; + } + +#ifdef RPR_DEBUG + elog(DEBUG2, "pos: " INT64_FORMAT " pattern: %s", + pos, pattern_str->data); +#endif + + /* look for matching pattern variable sequence */ +#ifdef RPR_DEBUG + elog(DEBUG1, "search_str_set started"); +#endif + num_matched_rows = search_str_set(pattern_str->data, + str_set, variable_pos); +#ifdef RPR_DEBUG + elog(DEBUG1, "search_str_set returns: %d", num_matched_rows); +#endif + variable_pos_discard(variable_pos); + string_set_discard(str_set); + + /* + * We are at the first row in the reduced frame. Save the number of + * matched rows as the number of rows in the reduced frame. + */ + if (num_matched_rows <= 0) + { + /* no match */ + register_reduced_frame_map(winstate, original_pos, RF_UNMATCHED); + } + else + { + register_reduced_frame_map(winstate, original_pos, RF_FRAME_HEAD); + + for (i = original_pos + 1; i < original_pos + num_matched_rows; i++) + { + register_reduced_frame_map(winstate, i, RF_SKIPPED); + } + } + + return; +} + +/* + * search_str_set + * Perform pattern matching using "pattern" against str_set. pattern is a + * regular expression derived from PATTERN clause. Note that the regular + * expression string is prefixed by '^' and followed by initials represented + * in a same way as str_set. str_set is a set of StringInfo. Each StringInfo + * has a string comprising initials of pattern variable strings being true in + * a row. The initials are one of [a-y], parallel to the order of variable + * names in DEFINE clause. Suppose DEFINE has variables START, UP and DOWN. If + * PATTERN has START, UP+ and DOWN, then the initials in PATTERN will be 'a', + * 'b' and 'c'. The "pattern" will be "^ab+c". + * + * variable_pos is an array representing the order of pattern variable string + * initials in PATTERN clause. For example initial 'a' potion is in + * variable_pos[0].pos[0] = 0. Note that if the pattern is "START UP DOWN UP" + * (UP appears twice), then "UP" (initial is 'b') has two position 1 and + * 3. Thus variable_pos for b is variable_pos[1].pos[0] = 1 and + * variable_pos[1].pos[1] = 3. + * + * Returns the longest number of the matching rows (greedy matching) if + * quatifier '+' or '*' is included in "pattern". + */ +static +int +search_str_set(char *pattern, StringSet * str_set, VariablePos * variable_pos) +{ +#define MAX_CANDIDATE_NUM 10000 /* max pattern match candidate size */ +#define FREEZED_CHAR 'Z' /* a pattern is freezed if it ends with the + * char */ +#define DISCARD_CHAR 'z' /* a pattern is not need to keep */ + + int set_size; /* number of rows in the set */ + int resultlen; + int index; + StringSet *old_str_set, + *new_str_set; + int new_str_size; + int len; + + set_size = string_set_get_size(str_set); + new_str_set = string_set_init(); + len = 0; + resultlen = 0; + + /* + * Generate all possible pattern variable name initials as a set of + * StringInfo named "new_str_set". For example, if we have two rows + * having "ab" (row 0) and "ac" (row 1) in the input str_set, new_str_set + * will have set of StringInfo "aa", "ac", "ba" and "bc" in the end. + */ +#ifdef RPR_DEBUG + elog(DEBUG1, "pattern: %s set_size: %d", pattern, set_size); +#endif + for (index = 0; index < set_size; index++) + { + StringInfo str; /* search target row */ + char *p; + int old_set_size; + int i; + +#ifdef RPR_DEBUG + elog(DEBUG1, "index: %d", index); +#endif + if (index == 0) + { + /* copy variables in row 0 */ + str = string_set_get(str_set, index); + p = str->data; + + /* + * Loop over each new pattern variable char. + */ + while (*p) + { + StringInfo new = makeStringInfo(); + + /* add pattern variable char */ + appendStringInfoChar(new, *p); + /* add new one to string set */ + string_set_add(new_str_set, new); +#ifdef RPR_DEBUG + elog(DEBUG1, "old_str: NULL new_str: %s", new->data); +#endif + p++; /* next pattern variable */ + } + } + else /* index != 0 */ + { + old_str_set = new_str_set; + new_str_set = string_set_init(); + str = string_set_get(str_set, index); + old_set_size = string_set_get_size(old_str_set); + + /* + * Loop over each rows in the previous result set. + */ + for (i = 0; i < old_set_size; i++) + { + StringInfo new; + char last_old_char; + int old_str_len; + StringInfo old = string_set_get(old_str_set, i); + + p = old->data; + old_str_len = strlen(p); + if (old_str_len > 0) + last_old_char = p[old_str_len - 1]; + else + last_old_char = '\0'; + + /* Is this old set freezed? */ + if (last_old_char == FREEZED_CHAR) + { + /* if shorter match. we can discard it */ + if ((old_str_len - 1) < resultlen) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "discard this old set because shorter match: %s", + old->data); +#endif + continue; + } + +#ifdef RPR_DEBUG + elog(DEBUG1, "keep this old set: %s", old->data); +#endif + + /* move the old set to new_str_set */ + string_set_add(new_str_set, old); + old_str_set->str_set[i] = NULL; + continue; + } + /* Can this old set be discarded? */ + else if (last_old_char == DISCARD_CHAR) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "discard this old set: %s", old->data); +#endif + continue; + } + +#ifdef RPR_DEBUG + elog(DEBUG1, "str->data: %s", str->data); +#endif + + /* + * loop over each pattern variable initial char in the input + * set. + */ + for (p = str->data; *p; p++) + { + /* + * Optimization. Check if the row's pattern variable + * initial character position is greater than or equal to + * the old set's last pattern variable initial character + * position. For example, if the old set's last pattern + * variable initials are "ab", then the new pattern + * variable initial can be "b" or "c" but can not be "a", + * if the initials in PATTERN is something like "a b c" or + * "a b+ c+" etc. This optimization is possible when we + * only allow "+" quantifier. + */ + if (variable_pos_compare(variable_pos, last_old_char, *p)) + { + /* copy source string */ + new = makeStringInfo(); + enlargeStringInfo(new, old->len + 1); + appendStringInfoString(new, old->data); + /* add pattern variable char */ + appendStringInfoChar(new, *p); +#ifdef RPR_DEBUG + elog(DEBUG1, "old_str: %s new_str: %s", + old->data, new->data); +#endif + + /* + * Adhoc optimization. If the first letter in the + * input string is the first and second position one + * and there's no associated quatifier '+', then we + * can dicard the input because there's no chace to + * expand the string further. + * + * For example, pattern "abc" cannot match "aa". + */ +#ifdef RPR_DEBUG + elog(DEBUG1, "pattern[1]:%c pattern[2]:%c new[0]:%c new[1]:%c", + pattern[1], pattern[2], new->data[0], new->data[1]); +#endif + if (pattern[1] == new->data[0] && + pattern[1] == new->data[1] && + pattern[2] != '+' && + pattern[1] != pattern[2]) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "discard this new data: %s", + new->data); +#endif + pfree(new->data); + pfree(new); + continue; + } + + /* add new one to string set */ + string_set_add(new_str_set, new); + } + else + { + /* + * We are freezing this pattern string. Since there's + * no chance to expand the string further, we perform + * pattern matching against the string. If it does not + * match, we can discard it. + */ + len = do_pattern_match(pattern, old->data); + + if (len <= 0) + { + /* no match. we can discard it */ + continue; + } + + else if (len <= resultlen) + { + /* shorter match. we can discard it */ + continue; + } + else + { + /* match length is the longest so far */ + + int new_index; + + /* remember the longest match */ + resultlen = len; + + /* freeze the pattern string */ + new = makeStringInfo(); + enlargeStringInfo(new, old->len + 1); + appendStringInfoString(new, old->data); + /* add freezed mark */ + appendStringInfoChar(new, FREEZED_CHAR); +#ifdef RPR_DEBUG + elog(DEBUG1, "old_str: %s new_str: %s", old->data, new->data); +#endif + string_set_add(new_str_set, new); + + /* + * Search new_str_set to find out freezed entries + * that have shorter match length. Mark them as + * "discard" so that they are discarded in the + * next round. + */ + + /* new_index_size should be the one before */ + new_str_size = + string_set_get_size(new_str_set) - 1; + + /* loop over new_str_set */ + for (new_index = 0; new_index < new_str_size; + new_index++) + { + char new_last_char; + int new_str_len; + + new = string_set_get(new_str_set, new_index); + new_str_len = strlen(new->data); + if (new_str_len > 0) + { + new_last_char = + new->data[new_str_len - 1]; + if (new_last_char == FREEZED_CHAR && + (new_str_len - 1) <= len) + { + /* + * mark this set to discard in the + * next round + */ + appendStringInfoChar(new, DISCARD_CHAR); +#ifdef RPR_DEBUG + elog(DEBUG1, "add discard char: %s", new->data); +#endif + } + } + } + } + } + } + } + /* we no longer need old string set */ + string_set_discard(old_str_set); + } + } + + /* + * Perform pattern matching to find out the longest match. + */ + new_str_size = string_set_get_size(new_str_set); +#ifdef RPR_DEBUG + elog(DEBUG1, "new_str_size: %d", new_str_size); +#endif + len = 0; + resultlen = 0; + + for (index = 0; index < new_str_size; index++) + { + StringInfo s; + + s = string_set_get(new_str_set, index); + if (s == NULL) + continue; /* no data */ + +#ifdef RPR_DEBUG + elog(DEBUG1, "target string: %s", s->data); +#endif + len = do_pattern_match(pattern, s->data); + if (len > resultlen) + { + /* remember the longest match */ + resultlen = len; + + /* + * If the size of result set is equal to the number of rows in the + * set, we are done because it's not possible that the number of + * matching rows exceeds the number of rows in the set. + */ + if (resultlen >= set_size) + break; + } + } + + /* we no longer need new string set */ + string_set_discard(new_str_set); + + return resultlen; +} + +/* + * do_pattern_match + * perform pattern match using pattern against encoded_str. + * returns matching number of rows if matching is succeeded. + * Otherwise returns 0. + */ +static +int +do_pattern_match(char *pattern, char *encoded_str) +{ + Datum d; + text *res; + char *substr; + int len = 0; + text *pattern_text, + *encoded_str_text; + + pattern_text = cstring_to_text(pattern); + encoded_str_text = cstring_to_text(encoded_str); + + /* + * We first perform pattern matching using regexp_instr, then call + * textregexsubstr to get matched substring to know how long the matched + * string is. That is the number of rows in the reduced window frame. The + * reason why we can't call textregexsubstr in the first place is, it + * errors out if pattern does not match. + */ + if (DatumGetInt32(DirectFunctionCall2Coll( + regexp_instr, DEFAULT_COLLATION_OID, + PointerGetDatum(encoded_str_text), + PointerGetDatum(pattern_text)))) + { + d = DirectFunctionCall2Coll(textregexsubstr, + DEFAULT_COLLATION_OID, + PointerGetDatum(encoded_str_text), + PointerGetDatum(pattern_text)); + if (d != 0) + { + res = DatumGetTextPP(d); + substr = text_to_cstring(res); + len = strlen(substr); + pfree(substr); + } + } + pfree(encoded_str_text); + pfree(pattern_text); + + return len; +} + +/* + * evaluate_pattern + * Evaluate expression associated with PATTERN variable vname. current_pos is + * relative row position in a frame (starting from 0). If vname is evaluated + * to true, initial letters associated with vname is appended to + * encode_str. result is out paramater representing the expression evaluation + * result is true of false. + *--------- + * Return values are: + * >=0: the last match absolute row position + * otherwise out of frame. + *--------- + */ +static +int64 +evaluate_pattern(WindowObject winobj, int64 current_pos, + char *vname, StringInfo encoded_str, bool *result) +{ + WindowAggState *winstate = winobj->winstate; + ExprContext *econtext = winstate->ss.ps.ps_ExprContext; + ListCell *lc1, + *lc2, + *lc3; + ExprState *pat; + Datum eval_result; + bool out_of_frame = false; + bool isnull; + TupleTableSlot *slot; + + forthree(lc1, winstate->defineVariableList, + lc2, winstate->defineClauseList, + lc3, winstate->defineInitial) + { + char initial; /* initial letter associated with vname */ + char *name = strVal(lfirst(lc1)); + + if (strcmp(vname, name)) + continue; + + initial = *(strVal(lfirst(lc3))); + + /* set expression to evaluate */ + pat = lfirst(lc2); + + /* get current, previous and next tuples */ + if (!get_slots(winobj, current_pos)) + { + out_of_frame = true; + } + else + { + /* evaluate the expression */ + eval_result = ExecEvalExpr(pat, econtext, &isnull); + if (isnull) + { + /* expression is NULL */ +#ifdef RPR_DEBUG + elog(DEBUG1, "expression for %s is NULL at row: " INT64_FORMAT, + vname, current_pos); +#endif + *result = false; + } + else + { + if (!DatumGetBool(eval_result)) + { + /* expression is false */ +#ifdef RPR_DEBUG + elog(DEBUG1, "expression for %s is false at row: " INT64_FORMAT, + vname, current_pos); +#endif + *result = false; + } + else + { + /* expression is true */ +#ifdef RPR_DEBUG + elog(DEBUG1, "expression for %s is true at row: " INT64_FORMAT, + vname, current_pos); +#endif + appendStringInfoChar(encoded_str, initial); + *result = true; + } + } + + slot = winstate->temp_slot_1; + if (slot != winstate->null_slot) + ExecClearTuple(slot); + slot = winstate->prev_slot; + if (slot != winstate->null_slot) + ExecClearTuple(slot); + slot = winstate->next_slot; + if (slot != winstate->null_slot) + ExecClearTuple(slot); + + break; + } + + if (out_of_frame) + { + *result = false; + return -1; + } + } + return current_pos; +} + +/* + * get_slots + * Get current, previous and next tuples. + * Returns false if current row is out of partition/full frame. + */ +static +bool +get_slots(WindowObject winobj, int64 current_pos) +{ + WindowAggState *winstate = winobj->winstate; + TupleTableSlot *slot; + int ret; + ExprContext *econtext; + + econtext = winstate->ss.ps.ps_ExprContext; + + /* set up current row tuple slot */ + slot = winstate->temp_slot_1; + if (!window_gettupleslot(winobj, current_pos, slot)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "current row is out of partition at:" INT64_FORMAT, + current_pos); +#endif + return false; + } + ret = row_is_in_frame(winstate, current_pos, slot); + if (ret <= 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "current row is out of frame at: " INT64_FORMAT, + current_pos); +#endif + ExecClearTuple(slot); + return false; + } + econtext->ecxt_outertuple = slot; + + /* for PREV */ + if (current_pos > 0) + { + slot = winstate->prev_slot; + if (!window_gettupleslot(winobj, current_pos - 1, slot)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "previous row is out of partition at: " INT64_FORMAT, + current_pos - 1); +#endif + econtext->ecxt_scantuple = winstate->null_slot; + } + else + { + ret = row_is_in_frame(winstate, current_pos - 1, slot); + if (ret <= 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "previous row is out of frame at: " INT64_FORMAT, + current_pos - 1); +#endif + ExecClearTuple(slot); + econtext->ecxt_scantuple = winstate->null_slot; + } + else + { + econtext->ecxt_scantuple = slot; + } + } + } + else + econtext->ecxt_scantuple = winstate->null_slot; + + /* for NEXT */ + slot = winstate->next_slot; + if (!window_gettupleslot(winobj, current_pos + 1, slot)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "next row is out of partiton at: " INT64_FORMAT, + current_pos + 1); +#endif + econtext->ecxt_innertuple = winstate->null_slot; + } + else + { + ret = row_is_in_frame(winstate, current_pos + 1, slot); + if (ret <= 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "next row is out of frame at: " INT64_FORMAT, + current_pos + 1); +#endif + ExecClearTuple(slot); + econtext->ecxt_innertuple = winstate->null_slot; + } + else + econtext->ecxt_innertuple = slot; + } + return true; +} + +/* + * pattern_initial + * Return pattern variable initial character + * matching with pattern variable name vname. + * If not found, return 0. + */ +static +char +pattern_initial(WindowAggState *winstate, char *vname) +{ + char initial; + char *name; + ListCell *lc1, + *lc2; + + forboth(lc1, winstate->defineVariableList, + lc2, winstate->defineInitial) + { + name = strVal(lfirst(lc1)); /* DEFINE variable name */ + initial = *(strVal(lfirst(lc2))); /* DEFINE variable initial */ + + + if (!strcmp(name, vname)) + return initial; /* found */ + } + return 0; +} + +/* + * string_set_init + * Create dynamic set of StringInfo. + */ +static +StringSet * string_set_init(void) +{ +/* Initial allocation size of str_set */ +#define STRING_SET_ALLOC_SIZE 1024 + + StringSet *string_set; + Size set_size; + + string_set = palloc0(sizeof(StringSet)); + string_set->set_index = 0; + set_size = STRING_SET_ALLOC_SIZE; + string_set->str_set = palloc(set_size * sizeof(StringInfo)); + string_set->set_size = set_size; + + return string_set; +} + +/* + * string_set_add + * Add StringInfo str to StringSet string_set. + */ +static +void +string_set_add(StringSet * string_set, StringInfo str) +{ + Size set_size; + + set_size = string_set->set_size; + if (string_set->set_index >= set_size) + { + set_size *= 2; + string_set->str_set = repalloc(string_set->str_set, + set_size * sizeof(StringInfo)); + string_set->set_size = set_size; + } + + string_set->str_set[string_set->set_index++] = str; + + return; +} + +/* + * string_set_get + * Returns StringInfo specified by index. + * If there's no data yet, returns NULL. + */ +static +StringInfo +string_set_get(StringSet * string_set, int index) +{ + /* no data? */ + if (index == 0 && string_set->set_index == 0) + return NULL; + + if (index < 0 || index >= string_set->set_index) + elog(ERROR, "invalid index: %d", index); + + return string_set->str_set[index]; +} + +/* + * string_set_get_size + * Returns the size of StringSet. + */ +static +int +string_set_get_size(StringSet * string_set) +{ + return string_set->set_index; +} + +/* + * string_set_discard + * Discard StringSet. + * All memory including StringSet itself is freed. + */ +static +void +string_set_discard(StringSet * string_set) +{ + int i; + + for (i = 0; i < string_set->set_index; i++) + { + StringInfo str = string_set->str_set[i]; + + if (str) + { + pfree(str->data); + pfree(str); + } + } + pfree(string_set->str_set); + pfree(string_set); +} + +/* + * variable_pos_init + * Create and initialize variable postion structure + */ +static +VariablePos * variable_pos_init(void) +{ + VariablePos *variable_pos; + + variable_pos = palloc(sizeof(VariablePos) * NUM_ALPHABETS); + MemSet(variable_pos, -1, sizeof(VariablePos) * NUM_ALPHABETS); + return variable_pos; +} + +/* + * variable_pos_register + * Register pattern variable whose initial is initial into postion index. + * pos is position of initial. + * If pos is already registered, register it at next empty slot. + */ +static +void +variable_pos_register(VariablePos * variable_pos, char initial, int pos) +{ + int index = initial - 'a'; + int slot; + int i; + + if (pos < 0 || pos > NUM_ALPHABETS) + elog(ERROR, "initial is not valid char: %c", initial); + + for (i = 0; i < NUM_ALPHABETS; i++) + { + slot = variable_pos[index].pos[i]; + if (slot < 0) + { + /* empty slot found */ + variable_pos[index].pos[i] = pos; + return; + } + } + elog(ERROR, "no empty slot for initial: %c", initial); +} + +/* + * variable_pos_compare + * Returns true if initial1 can be followed by initial2 + */ +static +bool +variable_pos_compare(VariablePos * variable_pos, char initial1, char initial2) +{ + int index1, + index2; + int pos1, + pos2; + + for (index1 = 0;; index1++) + { + pos1 = variable_pos_fetch(variable_pos, initial1, index1); + if (pos1 < 0) + break; + + for (index2 = 0;; index2++) + { + pos2 = variable_pos_fetch(variable_pos, initial2, index2); + if (pos2 < 0) + break; + if (pos1 <= pos2) + return true; + } + } + return false; +} + +/* + * variable_pos_fetch + * Fetch position of pattern variable whose initial is initial, and whose index + * is index. If no postion was registered by initial, index, returns -1. + */ +static +int +variable_pos_fetch(VariablePos * variable_pos, char initial, int index) +{ + int pos = initial - 'a'; + + if (pos < 0 || pos > NUM_ALPHABETS) + elog(ERROR, "initial is not valid char: %c", initial); + + if (index < 0 || index > NUM_ALPHABETS) + elog(ERROR, "index is not valid: %d", index); + + return variable_pos[pos].pos[index]; +} + +/* + * variable_pos_discard + * Discard VariablePos + */ +static +void +variable_pos_discard(VariablePos * variable_pos) +{ + pfree(variable_pos); +} diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c index 095de7741d..c5bd28ce19 100644 --- a/src/backend/utils/adt/windowfuncs.c +++ b/src/backend/utils/adt/windowfuncs.c @@ -13,6 +13,9 @@ */ #include "postgres.h" +#include "catalog/pg_collation_d.h" +#include "executor/executor.h" +#include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "nodes/supportnodes.h" #include "utils/builtins.h" @@ -37,11 +40,19 @@ typedef struct int64 remainder; /* (total rows) % (bucket num) */ } ntile_context; +/* + * rpr process information. + * Used for AFTER MATCH SKIP PAST LAST ROW + */ +typedef struct SkipContext +{ + int64 pos; /* last row absolute position */ +} SkipContext; + static bool rank_up(WindowObject winobj); static Datum leadlag_common(FunctionCallInfo fcinfo, bool forward, bool withoffset, bool withdefault); - /* * utility routine for *_rank functions. */ @@ -674,7 +685,7 @@ window_last_value(PG_FUNCTION_ARGS) bool isnull; result = WinGetFuncArgInFrame(winobj, 0, - 0, WINDOW_SEEK_TAIL, true, + 0, WINDOW_SEEK_TAIL, false, &isnull, NULL); if (isnull) PG_RETURN_NULL(); @@ -714,3 +725,25 @@ window_nth_value(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * prev + * Dummy function to invoke RPR's navigation operator "PREV". + * This is *not* a window function. + */ +Datum +window_prev(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(PG_GETARG_DATUM(0)); +} + +/* + * next + * Dummy function to invoke RPR's navigation operation "NEXT". + * This is *not* a window function. + */ +Datum +window_next(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(PG_GETARG_DATUM(0)); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 9c120fc2b7..81e4580b13 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10456,6 +10456,12 @@ { oid => '3114', descr => 'fetch the Nth row value', proname => 'nth_value', prokind => 'w', prorettype => 'anyelement', proargtypes => 'anyelement int4', prosrc => 'window_nth_value' }, +{ oid => '6122', descr => 'previous value', + proname => 'prev', provolatile => 's', prorettype => 'anyelement', + proargtypes => 'anyelement', prosrc => 'window_prev' }, +{ oid => '6123', descr => 'next value', + proname => 'next', provolatile => 's', prorettype => 'anyelement', + proargtypes => 'anyelement', prosrc => 'window_next' }, # functions for range types { oid => '3832', descr => 'I/O', diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 444a5f0fd5..15d8ac4c1e 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -2477,6 +2477,11 @@ typedef enum WindowAggStatus * tuples during spool */ } WindowAggStatus; +#define RF_NOT_DETERMINED 0 +#define RF_FRAME_HEAD 1 +#define RF_SKIPPED 2 +#define RF_UNMATCHED 3 + typedef struct WindowAggState { ScanState ss; /* its first field is NodeTag */ @@ -2525,6 +2530,19 @@ typedef struct WindowAggState int64 groupheadpos; /* current row's peer group head position */ int64 grouptailpos; /* " " " " tail position (group end+1) */ + /* these fields are used in Row pattern recognition: */ + RPSkipTo rpSkipTo; /* Row Pattern Skip To type */ + List *patternVariableList; /* list of row pattern variables names + * (list of String) */ + List *patternRegexpList; /* list of row pattern regular expressions + * ('+' or ''. list of String) */ + List *defineVariableList; /* list of row pattern definition + * variables (list of String) */ + List *defineClauseList; /* expression for row pattern definition + * search conditions ExprState list */ + List *defineInitial; /* list of row pattern definition variable + * initials (list of String) */ + MemoryContext partcontext; /* context for partition-lifespan data */ MemoryContext aggcontext; /* shared context for aggregate working data */ MemoryContext curaggcontext; /* current aggregate's working data */ @@ -2561,6 +2579,18 @@ typedef struct WindowAggState TupleTableSlot *agg_row_slot; TupleTableSlot *temp_slot_1; TupleTableSlot *temp_slot_2; + + /* temporary slots for RPR */ + TupleTableSlot *prev_slot; /* PREV row navigation operator */ + TupleTableSlot *next_slot; /* NEXT row navigation operator */ + TupleTableSlot *null_slot; /* all NULL slot */ + + /* + * Each byte corresponds to a row positioned at absolute its pos in + * partition. See above definition for RF_* + */ + char *reduced_frame_map; + int64 alloc_sz; /* size of the map */ } WindowAggState; /* ---------------- -- 2.25.1 ----Next_Part(Thu_Feb_29_09_19_54_2024_640)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v14-0006-Row-pattern-recognition-patch-docs.patch" ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
end of thread, other threads:[~2026-05-23 14:09 UTC | newest] Thread overview: 330+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2024-02-28 13:59 [PATCH v14 5/8] Row pattern recognition patch (executor). Tatsuo Ishii <ishii@postgresql.org> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox